##################################################################################################################
# Code for Results in Appendix Section A7
##################################################################################################################
write.table(data.frame(Description=c("","","","Appendix Section A7: Evidence on the Correct Disclosure of Hours Worked")),row.names=F,col.names=F,quote=F)

##################################################################################################################
# To run, please set the working directory to match the location of the folder containing all replication files.
##################################################################################################################
#setwd("Replication_PSRM_ST24")

#if not yet created, create "output" folder
if("output"%in%list.files()==F){dir.create("output")}

#Clear environment
rm(list=ls());gc();gc();gc();gc()

#If not yet installed, install packages
if(length(find.package("dplyr",quiet=T))==0){install.packages("dplyr")}

#Load package
library(dplyr)

#Set seed for simulations
set.seed(seed=539206723)

#Set locale
Sys.setlocale("LC_ALL", 'en_US.UTF-8')

#Load main data set
write.table(data.frame(Description=c("","Loading data...")),row.names=F,col.names=F,quote=F)

loball5ciA<-readRDS("Data/PSRM_main_dataset.rds")
df10<-readRDS("Data/Appendix_A7/lobbyist_time_reports.rds")
dfai<-readRDS("Data/Appendix_A7/committee_hearing_appearances.rds")


mmi<-merge(dfai,df10,by.x=c("new.lobbyist.name","new.merged.name","date3","date2"),by.y=c("lobbyist","common.name","date3","date2"))
#in merging, df10 lobbyist becomes new.lobbyist.name
#in merging, df10 common.name becomes new.merged.name
#so the remaining lobbyist and common.name come from coms/dfai

#manually change names
mmi$new.lobbyist.name[which(mmi$new.lobbyist.name=="Mark Grapentine, JD")]<-"Mark Grapentine"
mmi$new.lobbyist.name[which(mmi$new.lobbyist.name=="JD Mark Grapentine")]<-"Mark Grapentine"
mmi$new.lobbyist.name[which(mmi$new.lobbyist.name=="JD, Mark Grapentine")]<-"Mark Grapentine"

#merge appearances+time sheet with lobbyist data to get type
mmi2<-merge(mmi,unique(loball5ciA[,c("new.lobbyist.name","new.merged.name","type","time_period")]),by=c("new.lobbyist.name","new.merged.name","time_period"),all.x=T)

#manually add/correct missing/others
mmi2$type[grep("Robert Phillips, M\\. D\\.",mmi2$new.lobbyist.name)]<-"inhouse"
mmi2$type[grep("Mark Grapentine",mmi2$new.lobbyist.name)]<-"inhouse"
mmi2$type[grep("Nichole Yunk",mmi2$new.lobbyist.name)]<-"inhouse"
mmi2$type[grep("Stephen Einhorn",mmi2$new.lobbyist.name)]<-"inhouse"
mmi2$type[grep("Stephen Einhorn",mmi2$new.lobbyist.name)]<-"inhouse"
mmi2$type[grep("Richard Staff",mmi2$new.lobbyist.name)]<-"contract"
mmi2$type[grep("A\\.J\\. Wilson",mmi2$new.lobbyist.name)]<-"contract"
mmi2$type[intersect(grep("Dennis Puzz\\, jr",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Forest County Potawatomi Community"))]<-"contract"
mmi2$type[intersect(grep("Gerard Deschane",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Wisconsin Builders Association"))]<-"inhouse"
mmi2$type[intersect(grep("Lynn Morgan",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Waste Management Inc"))]<-"contract"
mmi2$type[intersect(grep("T Michael Bolger\\, J\\.D\\.",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Medical College of Wisconsin"))]<-"inhouse"
mmi2$type[intersect(grep("Alice O'Connor",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Special Olympics Wisconsin"))]<-"inhouse"
mmi2$type[intersect(grep("David Graves",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Wisconsin Sheriffs and Deputy Sheriffs Association"))]<-"inhouse"
mmi2$type[intersect(grep("Dennis Oulahan",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Milwaukee Teachers Education Association"))]<-"inhouse"
mmi2$type[intersect(grep("Christine Neumann-Ortiz",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Voces de la Frontera"))]<-"inhouse"
mmi2$type[intersect(grep("Amy Masek",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Wisconsin Early Autism Project Inc, The"))]<-"inhouse"
mmi2$type[intersect(grep("Ellen Henningsen",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Coalition of Wisconsin Aging Groups Inc"))]<-"inhouse"    
mmi2$type[intersect(grep("James J Kinsman Jr",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Brotherhood of Locomotive Engineers"))]<-"inhouse"                        
mmi2$type[intersect(grep("Jamie Aulik",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Wisconsin County Clerks Association"))]<-"inhouse"                        
mmi2$type[intersect(grep("Jeanne Benink",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="AARP"))]<-"inhouse"     
mmi2$type[intersect(grep("Maliyakal John",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="WiSys Technology Foundation, Inc."))]<-"inhouse"                          
mmi2$type[intersect(grep("Mark B Hazelbaker",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Wisconsin Association of Campground Owners"))]<-"contract"    
mmi2$type[intersect(grep("Matthew Guidry",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="United Council of UW Students"))]<-"inhouse"                              
mmi2$type[intersect(grep("Michael Vickerman",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="RENEW Wisconsin"))]<-"inhouse"                                            
mmi2$type[intersect(grep("Mitchell Hagopian",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Disability Rights Wisconsin"))]<-"inhouse"                                
mmi2$type[intersect(grep("Nicole Juan",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="United Council of UW Students Inc"))]<-"inhouse"                          
mmi2$type[intersect(grep("Todd Cohn",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="National Association of Professional Employer Organizations"))]<-"inhouse"	
mmi2$type[intersect(grep("Timothy J Elverman",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Pawn America Wisconsin LLC"))]<-"contract"	
mmi2$type[intersect(grep("John Alberts",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="St Croix Chippewa Indians of Wisconsin"))]<-"contract"	
mmi2$type[intersect(grep("Heather MacKinnon",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Wisconsin Bankers Association"))]<-"inhouse"	
mmi2$type[intersect(grep("Cedric Lawson",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="United Council of UW Students Inc"))]<-"inhouse"	
mmi2$type[intersect(grep("Robert J Kellerman",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Greater Wisconsin Agency on Aging Resources"))]<-"inhouse"	
mmi2$type[intersect(grep("Shawn W Pfaff",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Dairy Business Association"))]<-"contract"	
mmi2$type[intersect(grep("Tamara Jackson",mmi2$new.lobbyist.name),which(mmi2$new.merged.name=="Wisconsin Association of Lakes Inc"))]<-"inhouse"	


write.table(data.frame(Description=c("","Results:")),row.names=F,col.names=F,quote=F)
write.table(data.frame(Description=c("","Number of observations of lobbyist-client-hearing appearances:")),row.names=F,col.names=F,quote=F)
print(nrow(mmi2)) #6045
write.table(data.frame(Description=c("","Breakdown by Lobbyist Type:")),row.names=F,col.names=F,quote=F)
print(table(mmi2$type,useNA="always"))
#contract  inhouse     <NA> 
#   1136     4909        0  

write.table(data.frame(Description=c("","Percentage of committee hearing appearances for which lobbyists reported non-zero hours for relevant client:")),row.names=F,col.names=F,quote=F)
print(round(length(which(mmi2$d_any>0))/nrow(mmi2) ,3)*100) #0.916
write.table(data.frame(Description=c("","Percentage for contract lobbyists:")),row.names=F,col.names=F,quote=F)
print(round(length(which(mmi2$d_any>0&mmi2$type=="contract"))/nrow(mmi2[which(mmi2$type=="contract"),]),3)*100) #0.902
write.table(data.frame(Description=c("","Percentage for in-house lobbyists:")),row.names=F,col.names=F,quote=F)
print(round(length(which(mmi2$d_any>0&mmi2$type=="inhouse"))/nrow(mmi2[which(mmi2$type=="inhouse"),]),3)*100) #0.92

#simulation
write.table(data.frame(Description=c("","Starting simulations:")),row.names=F,col.names=F,quote=F)
mmilist<-list()
df10sim<-df10
for(i in 1:200){
  nname=paste("sample",i,sep="")
  df10sim<-df10sim%>%
    group_by(common.name,lobbyist,time_period)%>%
    mutate(!!nname:=sample(seq(min(date3),max(date3),1),length(seq(min(date3),max(date3),1))))
}
mmisim<-list()
mins<-which(colnames(df10sim)=="sample1")
maxs<-which(colnames(df10sim)=="sample200")
for(i in 1:200){
  delc<-setdiff(seq(mins,maxs,1),mins+i-1)
  mmisim[[i]]<-merge(dfai,df10sim[,-delc],by.x=c("new.lobbyist.name","new.merged.name","date3"),by.y=c("lobbyist","common.name",colnames(df10sim)[c(mins+i-1)]))
  #corrections as before
  mmisim[[i]]$new.lobbyist.name[which(mmisim[[i]]$new.lobbyist.name=="Mark Grapentine, JD")]<-"Mark Grapentine"
  mmisim[[i]]$new.lobbyist.name[which(mmisim[[i]]$new.lobbyist.name=="JD Mark Grapentine")]<-"Mark Grapentine"
  mmisim[[i]]$new.lobbyist.name[which(mmisim[[i]]$new.lobbyist.name=="JD, Mark Grapentine")]<-"Mark Grapentine"
  
  mmisim[[i]]<-merge(mmisim[[i]],unique(loball5ciA[,c("new.lobbyist.name","new.merged.name","type","time_period")]),by=c("new.lobbyist.name","new.merged.name","time_period"),all.x=T)
  #corrections as before
  mmisim[[i]]$type[grep("Robert Phillips, M\\. D\\.",mmisim[[i]]$new.lobbyist.name)]<-"inhouse"
  mmisim[[i]]$type[grep("Mark Grapentine",mmisim[[i]]$new.lobbyist.name)]<-"inhouse"
  mmisim[[i]]$type[grep("Nichole Yunk",mmisim[[i]]$new.lobbyist.name)]<-"inhouse"
  mmisim[[i]]$type[grep("Stephen Einhorn",mmisim[[i]]$new.lobbyist.name)]<-"inhouse"
  mmisim[[i]]$type[grep("Stephen Einhorn",mmisim[[i]]$new.lobbyist.name)]<-"inhouse"
  mmisim[[i]]$type[grep("Richard Staff",mmisim[[i]]$new.lobbyist.name)]<-"contract"
  mmisim[[i]]$type[grep("A\\.J\\. Wilson",mmisim[[i]]$new.lobbyist.name)]<-"contract"
  
  mmisim[[i]]$type[intersect(grep("Dennis Puzz\\, jr",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Forest County Potawatomi Community"))]<-"contract"
  mmisim[[i]]$type[intersect(grep("Gerard Deschane",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Wisconsin Builders Association"))]<-"inhouse"
  mmisim[[i]]$type[intersect(grep("Lynn Morgan",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Waste Management Inc"))]<-"contract"
  mmisim[[i]]$type[intersect(grep("T Michael Bolger\\, J\\.D\\.",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Medical College of Wisconsin"))]<-"inhouse"
  mmisim[[i]]$type[intersect(grep("Alice O'Connor",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Special Olympics Wisconsin"))]<-"inhouse"
  
  mmisim[[i]]$type[intersect(grep("David Graves",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Wisconsin Sheriffs and Deputy Sheriffs Association"))]<-"inhouse"
  mmisim[[i]]$type[intersect(grep("Dennis Oulahan",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Milwaukee Teachers Education Association"))]<-"inhouse"
  mmisim[[i]]$type[intersect(grep("Christine Neumann-Ortiz",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Voces de la Frontera"))]<-"inhouse"
  mmisim[[i]]$type[intersect(grep("Amy Masek",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Wisconsin Early Autism Project Inc, The"))]<-"inhouse"
  mmisim[[i]]$type[intersect(grep("Ellen Henningsen",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Coalition of Wisconsin Aging Groups Inc"))]<-"inhouse"    
  mmisim[[i]]$type[intersect(grep("James J Kinsman Jr",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Brotherhood of Locomotive Engineers"))]<-"inhouse"                        
  mmisim[[i]]$type[intersect(grep("Jamie Aulik",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Wisconsin County Clerks Association"))]<-"inhouse"                        
  mmisim[[i]]$type[intersect(grep("Jeanne Benink",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="AARP"))]<-"inhouse"     
  mmisim[[i]]$type[intersect(grep("Maliyakal John",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="WiSys Technology Foundation, Inc."))]<-"inhouse"                          
  mmisim[[i]]$type[intersect(grep("Mark B Hazelbaker",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Wisconsin Association of Campground Owners"))]<-"contract"    
  mmisim[[i]]$type[intersect(grep("Matthew Guidry",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="United Council of UW Students"))]<-"inhouse"                              
  mmisim[[i]]$type[intersect(grep("Michael Vickerman",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="RENEW Wisconsin"))]<-"inhouse"                                            
  mmisim[[i]]$type[intersect(grep("Mitchell Hagopian",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Disability Rights Wisconsin"))]<-"inhouse"                                
  
  mmisim[[i]]$type[intersect(grep("Nicole Juan",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="United Council of UW Students Inc"))]<-"inhouse"                          
  mmisim[[i]]$type[intersect(grep("Todd Cohn",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="National Association of Professional Employer Organizations"))]<-"inhouse"	
  mmisim[[i]]$type[intersect(grep("Timothy J Elverman",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Pawn America Wisconsin LLC"))]<-"contract"	
  mmisim[[i]]$type[intersect(grep("John Alberts",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="St Croix Chippewa Indians of Wisconsin"))]<-"contract"	
  mmisim[[i]]$type[intersect(grep("Heather MacKinnon",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="Wisconsin Bankers Association"))]<-"inhouse"	
  mmisim[[i]]$type[intersect(grep("Cedric Lawson",mmisim[[i]]$new.lobbyist.name),which(mmisim[[i]]$new.merged.name=="United Council of UW Students Inc"))]<-"inhouse"	
  
  mmisim$type[intersect(grep("Robert J Kellerman",mmisim$new.lobbyist.name),which(mmisim$new.merged.name=="Greater Wisconsin Agency on Aging Resources"))]<-"inhouse"	
  mmisim$type[intersect(grep("Shawn W Pfaff",mmisim$new.lobbyist.name),which(mmisim$new.merged.name=="Dairy Business Association"))]<-"contract"	
  mmisim$type[intersect(grep("Tamara Jackson",mmisim$new.lobbyist.name),which(mmisim$new.merged.name=="Wisconsin Association of Lakes Inc"))]<-"inhouse"	
  
   if(i%%10==0){print(paste(i/2,"% of simulations complete",sep=""))}
}
mmi2sim<-list(all=list(),contract=list(),inhouse=list(),all_c=list(),contract_c=list(),inhouse_c=list())
for(i in 1:200){
  mmi2sim$all[[i]]<-length(which(mmisim[[i]]$d_any>0&is.na(mmisim[[i]]$type)==F))/length(which(is.na(mmisim[[i]]$type)==F))
  mmi2sim$contract[[i]]<-length(which(mmisim[[i]]$d_any>0&is.na(mmisim[[i]]$type)==F&mmisim[[i]]$type=="contract"))/length(which(is.na(mmisim[[i]]$type)==F&mmisim[[i]]$type=="contract"))
  mmi2sim$inhouse[[i]]<-length(which(mmisim[[i]]$d_any>0&is.na(mmisim[[i]]$type)==F&mmisim[[i]]$type=="inhouse"))/length(which(is.na(mmisim[[i]]$type)==F&mmisim[[i]]$type=="inhouse"))
}

write.table(data.frame(Description=c("","Figure A9: Evidence on the Propensity to Disclose Hours Worked")),row.names=F,col.names=F,quote=F)

png("output/PSRM_figA9.png",width=8,height=6,units="in",res=300)
par(mar=c(5,5,0.5,5))
plot(col="blue",lwd=2,density(unlist(mmi2sim$all)),xlim=c(0.3,0.95),main="",xlab="Proportion of Hearings Attended With Same-Day Non-Zero Lobbying Hours") 
lines(lwd=2,density(unlist(mmi2sim$contract)),xlim=c(0.35,0.9),col="red") 
lines(lwd=2,density(unlist(mmi2sim$inhouse)),xlim=c(0.35,0.9),col="orange") 
abline(lty=2,v=c(length(which(mmi2$d_communicating+mmi2$d_other>0&is.na(mmi2$type)==F))/length(which(is.na(mmi2$type)==F)),
                 length(which(mmi2$d_communicating+mmi2$d_other>0&is.na(mmi2$type)==F&mmi2$type=="inhouse"))/length(which(is.na(mmi2$type)==F&mmi2$type=="inhouse")),
                 length(which(mmi2$d_communicating+mmi2$d_other>0&is.na(mmi2$type)==F&mmi2$type=="contract"))/length(which(is.na(mmi2$type)==F&mmi2$type=="contract"))),
       col=c("blue","orange","red"),lwd=2)
legend(lwd=2,"top",legend=c("All, Simulations","All, Sample","Contract, Simulations","Contract, Sample","In-House, Simulations","In-House, Sample"),col=c("blue","blue","red","red","orange","orange"),lty=c(rep(c(1,2),3)))
dev.off()

write.table(data.frame(Description=c("(N.B. The file for this figure is saved as 'PSRM_figA9.png' in the 'output' folder.)")),row.names=F,col.names=F,quote=F)

#Include code again so that Figure A6 also shown in R if code file is run using source()
par(mar=c(5,5,0.5,5))
plot(col="blue",lwd=2,density(unlist(mmi2sim$all)),xlim=c(0.3,0.95),main="",xlab="Proportion of Hearings Attended With Same-Day Non-Zero Lobbying Hours") 
lines(lwd=2,density(unlist(mmi2sim$contract)),xlim=c(0.35,0.9),col="red") 
lines(lwd=2,density(unlist(mmi2sim$inhouse)),xlim=c(0.35,0.9),col="orange") 
abline(lty=2,v=c(length(which(mmi2$d_communicating+mmi2$d_other>0&is.na(mmi2$type)==F))/length(which(is.na(mmi2$type)==F)),
                 length(which(mmi2$d_communicating+mmi2$d_other>0&is.na(mmi2$type)==F&mmi2$type=="inhouse"))/length(which(is.na(mmi2$type)==F&mmi2$type=="inhouse")),
                 length(which(mmi2$d_communicating+mmi2$d_other>0&is.na(mmi2$type)==F&mmi2$type=="contract"))/length(which(is.na(mmi2$type)==F&mmi2$type=="contract"))),
       col=c("blue","orange","red"),lwd=2)
legend(lwd=2,"top",legend=c("All, Simulations","All, Sample","Contract, Simulations","Contract, Sample","In-House, Simulations","In-House, Sample"),col=c("blue","blue","red","red","orange","orange"),lty=c(rep(c(1,2),3)))


write.table(data.frame(Description=c("","Simulated median proportion of hearings attended with same-day non-zero hours for relevant client:")),row.names=F,col.names=F,quote=F)
write.table(data.frame(Description=c("","Overall:")),row.names=F,col.names=F,quote=F)
print(round(median(unlist(mmi2sim$all)),3)) # 0.399
write.table(data.frame(Description=c("","For contract lobbyists:")),row.names=F,col.names=F,quote=F)
print(round(median(unlist(mmi2sim$contract)),3)) #0.347
write.table(data.frame(Description=c("","For in-house lobbyists:")),row.names=F,col.names=F,quote=F)
print(round(median(unlist(mmi2sim$inhouse)),3)) #0.411
